#!/usr/bin/python
# -*- coding: UTF-8 -*-

#su - hdfs -s /bin/bash -c  "spark-submit --master yarn --deploy-mode cluster --executor-cores 1 --executor-memory 1g --py-files /opt/cloudera/pyspark/sql_conf.py   /opt/cloudera/pyspark/sparktest.py "
#spark2-submit --master yarn --deploy-mode cluster --executor-cores 5 --num-executors 5 --driver-memory 2g --executor-memory 4g --conf spark.default.parallelism=20  --conf spark.yarn.executor.memoryOverhead=7G  --py-files /home/kf_xy_xy_mn/xhxy_credit_data/gsxx/sql_conf.py,/home/kf_xy_xy_mn/xhxy_credit_data/SparkUtil.py /home/kf_xy_xy_mn/xhxy_credit_data/gsxx/gsxx.py
#spark2-submit --master yarn --deploy-mode cluster --executor-cores 2 --num-executors 5 --driver-memory 2g --executor-memory 4g --conf spark.default.parallelism=5 --conf spark.yarn.executor.memoryOverhead=7G --py-files /mnt/disk4/zjs/xy/prod/whole_conf_ods_dwi.py,/mnt/disk4/zjs/xy/prod/SparkUtil.py /mnt/disk4/zjs/xy/prod/whole_script_ods_dwi.py

import whole_conf_ods_dwi as whole
from SparkUtil import SparkUtil
import time

startTime = time.time()
if __name__=="__main__":
    sparkUtil = SparkUtil(startTime)
    sparkUtil.execDatabaseName('xy_dwi')
    #公司信息 14
    SparkUtil.loadData(sparkUtil,whole.t_eci_company,'t_eci_company')
    SparkUtil.loadData(sparkUtil,whole.t_eci_companyattr,'t_eci_companyattr')
    SparkUtil.loadData(sparkUtil,whole.t_eci_industry,'t_eci_industry')
    SparkUtil.loadData(sparkUtil,whole.t_eci_partner,'t_eci_partner')
    SparkUtil.loadData(sparkUtil,whole.t_eci_employee,'t_eci_employee')
    SparkUtil.loadData(sparkUtil,whole.t_investinfo,'t_investinfo')
    SparkUtil.loadData(sparkUtil,whole.t_stockrelation_legal_info,'t_stockrelation_legal_info')
    SparkUtil.loadData(sparkUtil,whole.t_stockrelation_invest_info,'t_stockrelation_invest_info')
    SparkUtil.loadData(sparkUtil,whole.t_stockrelation_office_info,'t_stockrelation_office_info')
    SparkUtil.loadData(sparkUtil,whole.t_eci_branch,'t_eci_branch')
    SparkUtil.loadData(sparkUtil,whole.t_company_location,'t_company_location')
    SparkUtil.loadData(sparkUtil,whole.t_company_abstract,'t_company_abstract')
    SparkUtil.loadData(sparkUtil,whole.t_eci_count,'t_eci_count')
    SparkUtil.loadData(sparkUtil,whole.t_eci_change,'t_eci_change')
    #法律诉讼14
    SparkUtil.loadData(sparkUtil,whole.t_zhixing, 't_zhixing')
    SparkUtil.loadData(sparkUtil,whole.t_shixin, 't_shixin')
    SparkUtil.loadData(sparkUtil,whole.t_judgement, 't_judgement')
    SparkUtil.loadData(sparkUtil,whole.t_judgement_detail, 't_judgement_detail')
    SparkUtil.loadData(sparkUtil,whole.t_court, 't_court')
    SparkUtil.loadData(sparkUtil,whole.t_courtnotice, 't_courtnotice')
    SparkUtil.loadData(sparkUtil,whole.t_delivery_notice, 't_delivery_notice')
    SparkUtil.loadData(sparkUtil,whole.t_register, 't_register')
    SparkUtil.loadData(sparkUtil,whole.t_register_detail, 't_register_detail')
    SparkUtil.loadData(sparkUtil,whole.t_eci_assistance, 't_eci_assistance')
    SparkUtil.loadData(sparkUtil,whole.t_eci_assistance_equity_freeze, 't_eci_assistance_equity_freeze')
    SparkUtil.loadData(sparkUtil,whole.t_eci_assistance_equity_unfreeze, 't_eci_assistance_equity_unfreeze')
    SparkUtil.loadData(sparkUtil,whole.t_eci_assistance_judicial_partners_change, 't_eci_assistance_judicial_partners_change')
    SparkUtil.loadData(sparkUtil,whole.t_lawsuit_count, 't_lawsuit_count')
    #经营风险26
    SparkUtil.loadData(sparkUtil,whole.t_eci_operateexception,'t_eci_operateexception')
    SparkUtil.loadData(sparkUtil,whole.t_eci_seriousviolation,'t_eci_seriousviolation')
    SparkUtil.loadData(sparkUtil,whole.t_eci_pledge,'t_eci_pledge')
    SparkUtil.loadData(sparkUtil,whole.t_eci_penalty,'t_eci_penalty')
    SparkUtil.loadData(sparkUtil,whole.t_eci_movablespledge,'t_eci_movablespledge')
    SparkUtil.loadData(sparkUtil,whole.t_eci_mpledge_pledge,'t_eci_mpledge_pledge')
    SparkUtil.loadData(sparkUtil,whole.t_eci_mpledge_pledgee,'t_eci_mpledge_pledgee')
    SparkUtil.loadData(sparkUtil,whole.t_eci_mpledge_guaranteed_cred_right,'t_eci_mpledge_guaranteed_cred_right')
    SparkUtil.loadData(sparkUtil,whole.t_eci_mpledge_guarantee,'t_eci_mpledge_guarantee')
    SparkUtil.loadData(sparkUtil,whole.t_eci_mpledge_change,'t_eci_mpledge_change')
    SparkUtil.loadData(sparkUtil,whole.t_eci_mpledge_cancel,'t_eci_mpledge_cancel')
    SparkUtil.loadData(sparkUtil,whole.t_eci_liquidation,'t_eci_liquidation')
    SparkUtil.loadData(sparkUtil,whole.t_eci_simplecancellation,'t_eci_simplecancellation')
    SparkUtil.loadData(sparkUtil,whole.t_eci_simplecancellation_dissent,'t_eci_simplecancellation_dissent')
    SparkUtil.loadData(sparkUtil,whole.t_eci_simplecancellation_cancellation_result,'t_eci_simplecancellation_cancellation_result')
    SparkUtil.loadData(sparkUtil,whole.t_tax_illegal_detail,'t_tax_illegal_detail')
    SparkUtil.loadData(sparkUtil,whole.t_land_mortgage_details,'t_land_mortgage_details')
    SparkUtil.loadData(sparkUtil,whole.t_publish_notice_detail,'t_publish_notice_detail')
    SparkUtil.loadData(sparkUtil,whole.t_tax_owe,'t_tax_owe')
    SparkUtil.loadData(sparkUtil,whole.t_tax_owe_detail,'t_tax_owe_detail')
    SparkUtil.loadData(sparkUtil,whole.t_credit_china_penalty,'t_credit_china_penalty')
    SparkUtil.loadData(sparkUtil,whole.t_credit_china_penalty_detail,'t_credit_china_penalty_detail')
    SparkUtil.loadData(sparkUtil,whole.t_env_punishment,'t_env_punishment')
    SparkUtil.loadData(sparkUtil,whole.t_env_punishment_detail,'t_env_punishment_detail')
    SparkUtil.loadData(sparkUtil,whole.t_auction,'t_auction')
    SparkUtil.loadData(sparkUtil,whole.t_business_risk_count,'t_business_risk_count')
    #经营状况30
    SparkUtil.loadData(sparkUtil,whole.t_eci_licens, 't_eci_licens')
    SparkUtil.loadData(sparkUtil,whole.t_eci_spotcheck, 't_eci_spotcheck')
    SparkUtil.loadData(sparkUtil,whole.t_tender, 't_tender')
    SparkUtil.loadData(sparkUtil,whole.t_import_export, 't_import_export')
    SparkUtil.loadData(sparkUtil,whole.t_import_export_detail, 't_import_export_detail')
    SparkUtil.loadData(sparkUtil,whole.t_land_publish, 't_land_publish')
    SparkUtil.loadData(sparkUtil,whole.t_land_publish_detail, 't_land_publish_detail')
    SparkUtil.loadData(sparkUtil,whole.t_land_purchase, 't_land_purchase')
    SparkUtil.loadData(sparkUtil,whole.t_land_purchase_detail, 't_land_purchase_detail')
    SparkUtil.loadData(sparkUtil,whole.t_land_transfer, 't_land_transfer')
    SparkUtil.loadData(sparkUtil,whole.t_land_transfer_detail, 't_land_transfer_detail')
    SparkUtil.loadData(sparkUtil,whole.t_bond_detail, 't_bond_detail')
    SparkUtil.loadData(sparkUtil,whole.t_credit_china_license, 't_credit_china_license')
    SparkUtil.loadData(sparkUtil,whole.t_credit_china_license_detail, 't_credit_china_license_detail')
    SparkUtil.loadData(sparkUtil,whole.t_taxcredit, 't_taxcredit')
    SparkUtil.loadData(sparkUtil,whole.t_recruitinfo, 't_recruitinfo')
    # SparkUtil.loadData(sparkUtil,whole.t_financing_tax,'t_financing_tax')
    SparkUtil.loadData(sparkUtil,whole.t_newsinfo, 't_newsinfo')
    SparkUtil.loadData(sparkUtil,whole.t_product, 't_product')
    SparkUtil.loadData(sparkUtil,whole.t_financing, 't_financing')
    SparkUtil.loadData(sparkUtil,whole.t_credit_rating_latest, 't_credit_rating_latest')
    SparkUtil.loadData(sparkUtil,whole.t_credit_rating_detail, 't_credit_rating_detail')
    SparkUtil.loadData(sparkUtil,whole.t_credit_rating_pdf, 't_credit_rating_pdf')
    SparkUtil.loadData(sparkUtil,whole.t_customer, 't_customer')
    SparkUtil.loadData(sparkUtil,whole.t_microblog_info, 't_microblog_info')
    SparkUtil.loadData(sparkUtil,whole.t_product_competitor, 't_product_competitor')
    SparkUtil.loadData(sparkUtil,whole.t_supplier, 't_supplier')
    SparkUtil.loadData(sparkUtil,whole.t_telecom_license_detail, 't_telecom_license_detail')
    SparkUtil.loadData(sparkUtil,whole.t_wechat, 't_wechat')
    SparkUtil.loadData(sparkUtil,whole.t_manage_count, 't_manage_count')
    #建筑资质12
    SparkUtil.loadData(sparkUtil,whole.t_building_qualification,'t_building_qualification')
    SparkUtil.loadData(sparkUtil,whole.t_building_person,'t_building_person')
    SparkUtil.loadData(sparkUtil,whole.t_building_person_reginfo,'t_building_person_reginfo')
    SparkUtil.loadData(sparkUtil,whole.t_building_person_achinfo,'t_building_person_achinfo')
    SparkUtil.loadData(sparkUtil,whole.t_building_person_changeinfo,'t_building_person_changeinfo')
    SparkUtil.loadData(sparkUtil,whole.t_building_project,'t_building_project')
    SparkUtil.loadData(sparkUtil,whole.t_building_project_cclist,'t_building_project_cclist')
    SparkUtil.loadData(sparkUtil,whole.t_building_project_tenderlist,'t_building_project_tenderlist')
    SparkUtil.loadData(sparkUtil,whole.t_building_project_drawchklist,'t_building_project_drawchklist')
    SparkUtil.loadData(sparkUtil,whole.t_building_project_conreclist,'t_building_project_conreclist')
    SparkUtil.loadData(sparkUtil,whole.t_building_project_consliclist,'t_building_project_consliclist')
    SparkUtil.loadData(sparkUtil,whole.t_building_project_finverlist,'t_building_project_finverlist')
    #企业年报13
    SparkUtil.loadData(sparkUtil,whole.t_annual_report,'t_annual_report')
    SparkUtil.loadData(sparkUtil,whole.t_annual_basicinfo,'t_annual_basicinfo')
    SparkUtil.loadData(sparkUtil,whole.t_annual_assets,'t_annual_assets')
    SparkUtil.loadData(sparkUtil,whole.t_annual_change,'t_annual_change')
    SparkUtil.loadData(sparkUtil,whole.t_annual_investinfo,'t_annual_investinfo')
    SparkUtil.loadData(sparkUtil,whole.t_annual_partner,'t_annual_partner')
    SparkUtil.loadData(sparkUtil,whole.t_annual_provideassurance,'t_annual_provideassurance')
    SparkUtil.loadData(sparkUtil,whole.t_annual_stockchange,'t_annual_stockchange')
    SparkUtil.loadData(sparkUtil,whole.t_annual_website,'t_annual_website')
    SparkUtil.loadData(sparkUtil,whole.t_annual_administrationlicense,'t_annual_administrationlicense')
    SparkUtil.loadData(sparkUtil,whole.t_annual_branch,'t_annual_branch')
    SparkUtil.loadData(sparkUtil,whole.t_annual_employee,'t_annual_employee')
    SparkUtil.loadData(sparkUtil,whole.t_annual_socialsecurity,'t_annual_socialsecurity')
    #上市信息13
    SparkUtil.loadData(sparkUtil,whole.t_ipo_company_title,'t_ipo_company_title')
    SparkUtil.loadData(sparkUtil,whole.t_ipo_company_info,'t_ipo_company_info')
    SparkUtil.loadData(sparkUtil,whole.t_ipo_company_senior_executive,'t_ipo_company_senior_executive')
    SparkUtil.loadData(sparkUtil,whole.t_ipo_company_core_theme,'t_ipo_company_core_theme')
    SparkUtil.loadData(sparkUtil,whole.t_ipo_company_important_dwdb,'t_ipo_company_important_dwdb')
    SparkUtil.loadData(sparkUtil,whole.t_ipo_company_important_sszc,'t_ipo_company_important_sszc')
    SparkUtil.loadData(sparkUtil,whole.t_ipo_company_important_wgcl,'t_ipo_company_important_wgcl')
    SparkUtil.loadData(sparkUtil,whole.t_ipo_company_income_statement,'t_ipo_company_income_statement')
    SparkUtil.loadData(sparkUtil,whole.t_ipo_company_prospectus,'t_ipo_company_prospectus')
    SparkUtil.loadData(sparkUtil,whole.t_ipo_company_main_indicator,'t_ipo_company_main_indicator')
    SparkUtil.loadData(sparkUtil,whole.t_ipo_company_balance_sheet,'t_ipo_company_balance_sheet')
    SparkUtil.loadData(sparkUtil,whole.t_ipo_company_cash_flow_statement,'t_ipo_company_cash_flow_statement')
    SparkUtil.loadData(sparkUtil,whole.t_ipo_company_toptenholder,'t_ipo_company_toptenholder')
    #知识产权7
    SparkUtil.loadData(sparkUtil,whole.t_certification,'t_certification')
    SparkUtil.loadData(sparkUtil,whole.t_copyrights_copyright,'t_copyrights_copyright')
    SparkUtil.loadData(sparkUtil,whole.t_copyrights_softwarecopyright,'t_copyrights_softwarecopyright')
    SparkUtil.loadData(sparkUtil,whole.t_patent,'t_patent')
    SparkUtil.loadData(sparkUtil,whole.t_tm,'t_tm')
    SparkUtil.loadData(sparkUtil,whole.t_website,'t_website')
    SparkUtil.loadData(sparkUtil,whole.t_intellectual_property_count,'t_intellectual_property_count')

    sparkUtil.distory()

    print("--------------xy_ods清洗到xy_dwi完成--------------")
    endTime = time.time()
    print("startTime", startTime, "--->endTime:", endTime)
    print('总共花费时间: {} hours'.format((endTime - startTime) / 3600))